require 'rubygems'
require 'progressbar'

module Extract
  
  class Extractor
    
    public
    
    def initialize(opt = {})
      @crawl_table = Object.const_get "#{opt[:service_name].capitalize}Crawl"
      @data_table = Object.const_get "#{opt[:service_name].capitalize}Data"
      @conditions = !opt[:conditions].nil? ? opt[:conditions] : "last_update IS NULL"
      @rules = opt[:rules] ? opt[:rules] : {}
    end
    
    def extract_all
      pbar = ProgressBar.new("#{@data_table.table_name}", 100)
      last_progress_value = 0
      progress_value = last_progress_value
      records_analyzed = 0
      count = @data_table.count_by_sql "SELECT count(*) FROM #{@data_table.table_name} WHERE #{@conditions}"
      
      @data_table.find_each(:conditions => @conditions) { |d|
        begin
          doc = open_doc d.crawl.body
          @rules.collect { |rule| rule.extract doc, d }
          d.last_update = DateTime.now.strftime "%F %T"
          d.save
          
          records_analyzed += 1
          progress_value = sprintf("%d", records_analyzed.to_f / count * 100.0).to_i
          if  progress_value != last_progress_value then
            last_progress_value = progress_value
            pbar.set progress_value
          end
          
        rescue Exception => e
          p "EXCPTION BEGINNING"
          p d
          puts d.crawl.body
          puts e.inspect
          puts e.backtrace
          p "EXCPTION END"
        end
      }
      
      
      puts pbar.finish
    end
    
    def update_data_chunk
      conditions = "status = 200 AND id NOT IN (SELECT crawl_id FROM #{@data_table.table_name} WHERE crawl_id = #{@crawl_table.table_name}.id)"
      record_to_create = @crawl_table.count_by_sql "SELECT count(*) FROM #{@crawl_table.table_name} WHERE #{conditions}"
      
      pbar = ProgressBar.new("#{@data_table.table_name}", 100)
      last_progress_value = 0
      progress_value = last_progress_value
      records_analyzed = 0
      
      @crawl_table.find_each(:conditions => conditions) { |c|
        c.create_data
        
        records_analyzed += 1
        progress_value = sprintf("%d", records_analyzed.to_f / record_to_create * 100.0).to_i
        if  progress_value != last_progress_value then
          last_progress_value = progress_value
          pbar.set progress_value
        end
      }        
      
      puts pbar.finish
    end
    
    
  end
  
end  # module Extractor